package com.catmiao.spark.acc

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

/**
 * @title: Spark01_RDD_Dep
 * @projectName spark_study
 * @description: 广播变量
 * @author ChengMiao
 * @date 2024/2/28 23:20
 */
object Spark02_Bc {


  def main(args: Array[String]): Unit = {
    val sparkCon = new SparkConf().setMaster("local[*]").setAppName("bc")

    val sparkContext = new SparkContext(sparkCon)

    val rdd1 = sparkContext.makeRDD(List(
      ("a",1),
      ("b",2),
      ("c",3)
    ))

    val map = mutable.Map(
      ("a",5),
      ("b",6),
      ("c",7),
    )

    // join会导致数据量几何增长，并且会影响shuffle的性能，不推荐使用
//    val joinRdd: RDD[(String, (Int, Int))] = rdd1.join(rdd2)

    // 封装广播变量 【共享】
    val bc: Broadcast[mutable.Map[String, Int]] = sparkContext.broadcast(map)

    rdd1.map {
      case (w,c) => {
        // 方法广播变量
        val l: Int = bc.value.getOrElse(w,0)
        (w,(c,l))
      }
    }.collect().foreach(println)



    sparkContext.stop()
  }
}
